#pragma once

#include <string>
#include <vector>
#include <hash_map>
#include <regex>
#include <cassert>

#include "utf8string.h"
#include "lazy.h"
#include "ReservedWordDictionary.h"
#include "yyparser.h"

class Token
{
public:
    enum TokenType { invalid, reservedword, indentifier, number, string};

    Token(const std::string & source, const std::string::const_iterator position, std::size_t length, TokenType type)
		: _source(source), _position(position), _length(length), _type(type)
	{
	}

    int GetTokenType() const
    {
        switch(_type)
        {
        case invalid:
            return LEXERROR;
        case reservedword:
            return GetKeywordTypeFast();
        case indentifier:
            return T_IDENTIFIER;
        case number:
            return T_NUMBER_LITERAL;
        case string:
            return T_STRING_LITERAL;
        default:
            return LEXERROR;
        }
    }

	/// split text into different tokens with different types, ignoring comments
	static bool Tokenize(const std::string & text, std::vector<Token> & tokens)
	{
		bool ret = true;

		std::string::const_iterator it = text.begin();
		std::string::const_iterator end = text.end();
		std::string::const_iterator error_pos = text.end();

		while(it != end)
		{
			if (*it == ';')
			{
				tokens.push_back( Token(text, it, 1, reservedword) );
				++it;
			}
			else if (utf8string::iswhite(*it))
			{
				// skip extra white space charactor
				++it;
			}
			else if (*it == '"') // string start?
			{
				std::string::const_iterator token_start(it);
				utf8::iterator<std::string::const_iterator> utf8_it(it, it, end);
				utf8::iterator<std::string::const_iterator> utf8_end(end, it, end);

				++utf8_it;
				while(utf8_it != utf8_end && *utf8_it != utf8char('"')) ++utf8_it;

				if (utf8_it != utf8_end && *utf8_it == utf8char('"'))
				{
					it = utf8_it.base();
					++it;
					tokens.push_back( Token(text, token_start, (it- token_start), string) );
				}
				else
				{
					ret= false;
					break;
				}
			}
			else if (*it == '/') // comment?
			{
				++it;
				if (it == end)
				{
					error_pos = it;
					ret = false;
					break;
				}

				if (*it == '/')
				{
					++it;
					while(it!=end && *it!='\n') ++it;
				}
				else
				{
					tokens.push_back(Token(text, it-1, 1, reservedword));
				}
			}
			else if (utf8string::isalphabet(*it))
			{
				// maybe keyword / identifier
				std::string::const_iterator token_start = it;
				++it;
				while(it!=end && (utf8string::isalphabet(*it) || utf8string::isdigital(*it))) ++it;

                if (IsKeyword(token_start, it))
                {
                    tokens.push_back(Token(text, token_start, (it-token_start), reservedword));
                }
                else
                {
				    tokens.push_back(Token(text, token_start, (it-token_start), indentifier));
                }
			}
			else if (utf8string::isdigital(*it))
			{
				// maybe number;
                static std::tr1::regex number_rx("^[0-9]+");
                std::tr1::match_results<std::string::const_iterator> res;
                std::tr1::regex_search(it, end, res, number_rx);
                assert(res.empty() == false);
                assert(res[0].matched == true);
                tokens.push_back(Token(text, it, (res[0].second - it), number));
                it = res[0].second;
			}
			else if (*it == '=')
			{
                if (it+1!=end && *(it+1) == '=')
                {
				    tokens.push_back(Token(text, it, 2, reservedword));
                    it +=2;
                }
                else
                {
				    tokens.push_back(Token(text, it++, 1, reservedword));
                }
			}
			else if (*it == ',')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '*')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '(')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == ')')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '<')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '>')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '!' && it+1!=end && *(it+1)=='=' )
			{
				tokens.push_back(Token(text, it, 2, reservedword));
                it +=2;
			}
			else if (*it == '<' && it+1!=end && *(it+1)=='=' )
			{
				tokens.push_back(Token(text, it, 2, reservedword));
                it +=2;
			}
			else if (*it == '>' && it+1!=end && *(it+1)=='=' )
			{
				tokens.push_back(Token(text, it, 2, reservedword));
                it +=2;
			}
			else if (*it == '+')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
			else if (*it == '-')
			{
				tokens.push_back(Token(text, it++, 1, reservedword));
			}
		}
		return ret;
	}

private:
    // help identify number
    std::string::const_iterator isnumber(std::string::const_iterator start)
    {
    }

    // help identify string
    std::string::const_iterator isstring(std::string::const_iterator start)
    {
        std::string::const_iterator pos= start;
    }

    // help identify keyword
    static bool IsKeyword(std::string::const_iterator start, std::string::const_iterator end)
    {
        char buf[32];
        std::copy(start, end, buf);
        buf[end-start] = '\0';
        return reservedWordDictionary->IsIn(buf);
    }

    int GetKeywordTypeFast() const
    {
        char buf[32];
        std::copy_n(_position, _length, buf);
        buf[_length] = '\0';
        return reservedWordDictionary->GetType(buf);
    }

	const std::string::const_iterator _position;
	const std::size_t _length;
	const TokenType _type;
	const std::string & _source;

    static lazy<ReservedWordDictionary> reservedWordDictionary;
};